# encoding: utf-8
import json
import requests
import time
import datetime
from mysql.connector.errors import IntegrityError

import mysql_tools

# INSERT SQL
INSERT_SQL = u"""
    INSERT INTO suningyigou ({keys}) values ({values})
    """
# 请求频次，单位秒
REQUEST_RATE = 5 * 60
# REQUEST_RATE = 5
# 每页延迟，单位秒
PAGE_DEDLAY = 1

# MySQL同名字段名
STR_DATA_FIELD = [
    'screen_name',
    'profile_image_url',
    'profile_url',
    'verified_reason',
    'description',
    'remark',
    'text',
    'gender',
    'valid'
]
NUMBER_DATA_FIELD = [
    'user_id',
    'statuses_count',
    'verified_type',
    'mbtype',
    'ismember',
    'fansNum',
    'verified'  # bool --> 1 or 0
]
DATETIME_DATA_FIELD = [
    'create_time'
]

# 苏宁易购微博粉丝列表URL
URL = r"http://m.weibo.cn/page/json?containerid=1005051769962140_-_FANS&page={page}&count=2"
# 结束时间设置
START_TIME = datetime.time(hour=9)
END_TIME = datetime.time(hour=23, minute=59)
COUNT = 0

def sql_generator(response_dict):
    """
    插入语句生成
    args:response_dict
    """
    response_dict['user_id'] = response_dict['id']
    response_dict['verified'] = 1 if response_dict['verified'] is True else 0
    keys = []
    values = []
    for field in STR_DATA_FIELD:
        keys.append(field)
        values.append(u"{field}".format(field="'%s'" % response_dict.get(field, '')))
    for field in NUMBER_DATA_FIELD:
        keys.append(field)
        values.append(u'{field}'.format(field=response_dict.get(field, 0)))
    for field in DATETIME_DATA_FIELD:
        keys.append(field)
        values.append(u'{create_time}'.format(
            create_time="'%s'" % unicode(datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S"))))
    # print values

    return INSERT_SQL.format(keys=",".join(keys), values=",".join(values))


def is_duplicate(user_id, cursor):
    """
    查询数据库中user_id是否重复
    """
    cursor.execute("SELECT id FROM suningyigou WHERE user_id={}".format(user_id))
    rst = cursor.fetchall()
    return True if rst else False


def get_weibo_fans(count):
    # MySQL连接
    conn = mysql_tools.connect_mysql(password='njuptsun')
    cursor = mysql_tools.generate_cursor(conn)
    for i in xrange(1, 101):
        try:
            rst = requests.get(URL.format(page=str(i)))
        except:
            print 'continue: request timeout'
            continue
        print '>>>--->>> status:', rst.status_code, 'page:', i, '<<<---<<<'
        if process_data(rst, count, conn, cursor) is False:
            break
    conn.close()


def process_data(rst, count, conn, cursor):
    content = json.loads(rst.content)
    cards = content.get('cards', '')
    is_more_new_fans = True
    if cards:
        cardsgroup = cards[0].get('card_group')
        if cardsgroup:
            for card in cardsgroup:
                info = card['user']
                print info['id']
                # 唯一性校验利用设置user_id为UNIQUE来实现
                try:
                    cursor.execute(sql_generator(info))
                except IntegrityError:
                    print 'break: duplicate user_id'
                    is_more_new_fans = False
                    break
                except Exception, e:
                    print e
                    continue
                else:
                    print "insert success!"
                count += 1
                print "> fetch user count:", count, "<"
                conn.commit()
    time.sleep(PAGE_DEDLAY)
    return is_more_new_fans


if __name__ == "__main__":
    # count = 0
    while True:
        now = datetime.datetime.now()
        now = datetime.time(hour=now.hour, minute=now.minute)
        print '--- now:', now, '---'
        print '--- start_time:', START_TIME, '|| end_time:', END_TIME, '---'
        if START_TIME < now < END_TIME:
            get_weibo_fans(COUNT)
        print 'spider will restart after', REQUEST_RATE, 'seconds'
        time.sleep(REQUEST_RATE)
